import threading
import utils
import re
import os
import random
import time
import json
import requests
from login import Lighthouse, use_default_login
from urllib3.exceptions import InsecureRequestWarning

requests.urllib3.disable_warnings(InsecureRequestWarning)


def get_app_msg_id_list(sent_list):
    app_msg_id_list = []
    if not isinstance(sent_list, list):
        return app_msg_id_list
    for sent_item in sent_list:
        if isinstance(sent_item, dict) and 'appmsg_info' in sent_item and isinstance(sent_item['appmsg_info'], list):
            if len(sent_item['appmsg_info']):
                app_msg_id_list.append(sent_item['appmsg_info'][0]['appmsgid'])
    return app_msg_id_list


def merge_app_msg_other_info(sent_list, other_info):
    try:
        index_from = 0
        index_to = len(sent_list)
        for app_msg_entry in other_info:
            app_msg_items = app_msg_entry['appmsg_info']
            for index in range(index_from, index_to):
                app_msg_info = sent_list[index]['appmsg_info']
                if len(app_msg_info) and app_msg_info[0]['appmsgid'] == app_msg_items[0]['appmsgid']:
                    for sub_index, app_msg_sub_info in enumerate(app_msg_items):
                        app_msg_info[sub_index]['other_info'] = app_msg_sub_info
                    index_from = index
                    break
        return True
    except KeyError:
        return False


class WeChatClient:

    def __init__(self, login_service, url='https://mp.weixin.qq.com'):
        """
        通过moka登录接口获取公众号后台资料
        :param Lighthouse login_service:
        """
        self.url = url
        self.login = login_service
        self.session = utils.requests_retry_session()
        self.gh_id = None
        self.gh_nickname = None
        self.token = None
        self.index_url = None
        self.index_response = None
        self.violation_list = []

    def log_line(self, msg, gh_nickname=None, gh_id=None):
        line = '[{}]({}) {}'.format(gh_nickname if gh_nickname else self.gh_nickname,
                                    gh_id if gh_id else self.gh_id, msg)
        print(line)
        return line

    def log_err(self, err, logtype='log', gh_nickname=None, gh_id=None):
        utils.log_error(self.log_line(err, gh_nickname, gh_id), logtype)

    def debug_log(self, err, logtype='debug', gh_nickname=None, gh_id=None):
        utils.log_error(self.log_line(err, gh_nickname, gh_id), logtype)
        print(err)

    def detect_account_type(self):
        m = re.search(r'>((订阅|服务)号)</a>', self.index_response, re.MULTILINE)
        if m:
            return m.group(1)

    # 登录
    def enter(self, gh_id, gh_nickname, pwd):
        cookies = self.login.get_wx_cookie(gh_id)
        if cookies:
            self.log_line('获取Cookie成功', gh_nickname, gh_id)
            utils.set_cookies(self.session, cookies)
        else:
            self.log_line('一键登录状态异常，使用人工扫码登录',
                          gh_nickname=gh_nickname, gh_id=gh_id)
            return False
        response = self.session.get(url=self.url, verify=False)
        self.index_url = str(response.url)
        try:
            token = re.findall('token=(\\d+)', self.index_url)[0]
        except IndexError:
            self.log_err('微信后台登录失败，检查Cookie是否有效',
                            gh_nickname=gh_nickname, gh_id=gh_id)
            return False
        self.gh_id = gh_id
        self.gh_nickname = gh_nickname
        self.token = token
        self.index_response = response.content.decode('utf-8')
        self.log_line('后台登录成功，Token={}'.format(token))
        return True

    # 保存数据
    def save_data(self, data_file, data_date, data):
        with utils.open_write_data_file(self.gh_id, data_file, data_date) as f:
            json.dump(data, f)

    # 检测是否存在
    def detect_or_save(self, data_file, data_name, data_date, func, skip_exist=True, verbose=True):
        if skip_exist and utils.has_data_file(self.gh_id, data_file, data_date):
            if verbose:
                self.log_line('已存在{}，跳过'.format(data_name))
        else:
            if verbose:
                self.log_line('开始获取{}'.format(data_name))
            data = func(data_date)
            if data:
                if verbose:
                    self.log_line('获取{}成功'.format(data_name))
                self.save_data(data_file, data_date, data)
            else:
                self.log_err('获取{}失败,请重试,或单独爬取'.format(data_name))

    # 保存基础信息
    def save_basic_info(self, account, date):
        self.save_data('basic.json', date, account)

    def save_subscriber_info(self, date):
        resp = self.session.get(
            '{0}/misc/useranalysis?&begin_date={1}&end_date={1}&source=99999999&token={2}&lang=zh_CN&f=json&ajax=1&random={3}'.format(self.url, date, self.token, random.random()), verify=False)
        try:
            data = resp.json()
            return data['category_list'][0]['list'][0] if data['category_list'][0]['list'][0] else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_subscriber_attr(self, date):
        resp = self.session.get(
            '{0}/misc/useranalysis?action=attr&begin_date={1}&end_date={1}&token={2}&lang=zh_CN&f=json&ajax=1'.format(
                self.url, date, self.token), verify=False)
        try:
            data = resp.json()
            if data['load_done'] and data['user_portrait']['list']:
                return data['user_portrait']['list'][0]
            else:
                self.log_err("微信后台没有数据")
                return None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_messages_info(self, date):
        resp = self.session.get(
            '{0}/misc/appmsganalysis?action=report&type=daily&begin_date={1}&end_date={1}&token={2}'
            '&lang=zh_CN&f=json&ajax=1&random={3}'.format(self.url, date, self.token, random.random()), verify=False)
        try:
            data = resp.json()
            return {'item': data['item'], 'share': data['share_item']} if data['load_done'] else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_publisher_info(self, date):
        resp = self.session.get('{0}/promotion/publisher/publisher_stat?action=stat&lang=zh_CN&token={1}&appid=&_={2}'
                                .format(self.url, self.token, int(round(time.time() * 1000))), verify=False)
        try:
            data = resp.json()
            if data['base_resp']['ret'] == 0:
                for item in data['cost_list']['cost']:
                    if item['start_time'] == date:
                        return item
                else:
                    self.log_line('流量主开通但数据为空')
                    return {"view_count": 0, "click_count": 0, "cost": 0}
            elif data['base_resp']['ret'] == -1:
                self.log_line('未开通流量主')
                return 'EMPTY'
            else:
                self.log_err('流量主获取状态' + data['base_resp']['err_msg'])
                return None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    # 爬取违规数据
    def save_violation_info(self):
        self.violation_list = []
        resp = self.session.get('{0}/cgi-bin/sysnotify?token={1}&lang=zh_CN&f=json&ajax=1&random={2}&begin=0&'
                                'count=999999&status=0'.format(self.url, self.token, random.random()), verify=False)
        dates = set()
        try:
            data = resp.json()
            for item in data['List']:
                if 'Title' in item:
                    if re.search('(?:违规|责令处理|屏蔽)', item['Title']):
                        self.violation_list.append(item)
                        dates.add(time.strftime(
                            '%Y-%m-%d', time.localtime(item['UpdateTime'])))
            for date in dates:
                self.detect_or_save('violation-list.json', '违规处理', date, self.violation_filter_date,
                                    skip_exist=date != utils.get_today(), verbose=False)
        except (KeyError, ValueError, TypeError):
            self.log_err('获取违规失败')
            self.debug_log(resp.content.decode('utf-8'))

    def violation_filter_date(self, date):
        violations = []
        for item in self.violation_list:
            update_time = time.strftime(
                '%Y-%m-%d', time.localtime(item['UpdateTime']))
            if date == update_time:
                violations.append({
                    'Id': item['Id'],
                    'UpdateTime': update_time,
                    'Content': item['Content'],
                    'Title': item['Title']
                })
        return violations

    def save_message_items_info(self, app_msg_id_list):
        resp = self.session.get('{0}/cgi-bin/appmsgotherinfo?appmsgidlist={1}'
                                '&token={2}&token={2}&lang=zh_CN&f=json&ajax=1'
                                .format(self.url, ','.join(map(str, app_msg_id_list)), self.token), verify=False)
        try:
            data = resp.json()
            return data['sent_list'] if data['base_resp']['ret'] == 0 else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    def save_messages_detail_info(self, date):
        m = re.search(
            'wx.cgiData.mass_data=({.+});$', self.index_response, re.MULTILINE)
        if m:
            try:
                mass_data = json.loads(m.group(1))
                if 'sent_list' in mass_data:
                    other_info = self.save_message_items_info(
                        get_app_msg_id_list(mass_data['sent_list']))
                    if merge_app_msg_other_info(mass_data['sent_list'], other_info):
                        self.save_data('app-msg-info.json', date,
                                       mass_data['sent_list'])
                        self.log_line('获取图文详情列表成功')
                    else:
                        self.log_err('合并图文详情列表失败')
                else:
                    self.log_err('获取图文详情列表失败')
                    self.debug_log(mass_data)
            except ValueError:
                self.log_err('解析图文详情列表JSON失败')
                self.debug_log(m)
        else:
            self.log_err('未发现图文详情列表')

    def save_messages_all_info(self, date):
        resp = self.session.get('{0}/misc/appmsganalysis?action=all&begin_date={1}&end_date={1}&order_by=1'
                                '&order_direction=2&token={2}&lang=zh_CN&f=json&ajax=1&random={3}'
                                .format(self.url, date, self.token, random.random()), verify=False)
        try:
            data = resp.json()
            return {"total": json.loads(data['total_article_data'])['list'],
                    "summary": json.loads(data['article_summary_data'])['list']
                    } if data['base_resp']['ret'] == 0 else None
        except (KeyError, ValueError, TypeError):
            self.debug_log(resp.content)

    # 爬取广告主数据
    def save_advertiser_info(self):
        data = None
        res = self.session.get('{0}/promotion/advertiser_manager?action=get_client_status&token={1}&appid=&spid=&_={2}'
                               .format(self.url, self.token, int(round(time.time() * 1000))), verify=False)
        try:
            stat = res.json()
            if stat['mp_status'] == 1:
                resp = self.session.get('{0}/promotion/agency/checkMpAdMasterAuthorize?&with_account_info=1&token={1}&appid=&spid=&_={2}'
                                        .format(self.url, self.token, int(round(time.time() * 1000))), verify=False)
                try:
                    content = str(resp.content, encoding="utf-8")
                    content = content.strip("(").strip(")")
                    data = json.loads(content)
                    if data['agencies'] == []:
                        self.log_line('广告主开通但未绑定服务商')
                        data = '广告主开通但未绑定服务商'
                except:
                    self.debug_log(resp.content)
                    self.log_err('获取广告主数据失败')
            else:
                self.log_line('未开通广告主')
                data = '未开通广告主'
            self.save_data('app-ad-info.json', utils.get_today(), data)
        except:
            self.debug_log(res.content)
            self.log_err('获取广告主失败')

    # 爬取昵称等信息
    def get_nickname_when_not(self):
        resp = self.session.get('{0}/cgi-bin/settingpage?t=setting/index&action=index&token={1}&lang=zh_CN&f=json&ajax=1&random={2}'
                                .format(self.url, self.token, random.random()), verify=False)
        nickname = ''
        wx_id = ''
        gs = ''
        try:
            data = resp.json()
            nickname = data['setting_info']['nickname']['nickname']
            wx_id = data['setting_info']['username']
            gs = data['setting_info']['contractor_info']['name']
        except:
            self.debug_log(resp.content)

        return nickname, wx_id, gs

    # 爬取一段时间内的流量主信息
    def get_publisher_info(self, start_date, end_date):
        date_list = utils.get_day_interval_format(start_date, end_date)
        res = self.session.get(
            '{0}/promotion/publisher/publisher_stat?action=biz_ads_stat&page=1&page_size=999&start_date={1}&end_date={2}&slot=1&token={3}&appid=&spid=&_={4}'
            .format(self.url, start_date, end_date, self.token, int(round(time.time() * 1000)), verify=False)
        )

        try:
            data = res.json()
            if ('total_num' in data.keys()):
                if (data['total_num'] != 0):
                    for item in data['list']:
                        self.save_data('publisher_by_day.json', item['date'], item)
                else:
                    for date in date_list:
                        self.save_data('publisher_by_day.json', date, 0)
            else:
                for date in date_list:
                    self.save_data('publisher_by_day.json', date, '未开通')

        except (KeyError, ValueError, TypeError):
            self.debug_log(res.content)

    # 爬取迁移状态
    def get_transfer_info(self, date):
        res = self.session.get('{0}/acct/biztransferorder?action=index&token={1}&lang=zh_CN&f=json&ajax=1&random={2}'.format(self.url, self.token, random.random()), verify=False)
        try:
            data = res.json()
            if (data['base_resp']['err_msg'] == 'ok'):
                if ('order_list_json' in data):
                    content = '迁移中'
                else:
                    content = '可迁移'
            else:
                content = '不可迁移'
            self.save_data('transfer.json', date, content)
        except:
            self.debug_log(res.content)

    # 爬取认证发票信息
    def get_invoice_info(self, nick_name, month):
        os.makedirs('认证发票', exist_ok=True)
        res = self.session.get('{0}/acct/wxverifyorder?action=order_list&token={1}&lang=zh_CN&f=json&ajax=1&random={2}'.format(self.url, self.token, random.random()), verify=False)
        try:
            data = res.json()
            content = []
            if data['base_resp']['err_msg'] == 'ok':
                ticket = data['base_resp']['master_ticket']
                if 'order_list_json' in data:
                    order_list = json.loads(data['order_list_json'])
                    order_content = []
                    for order in order_list['list']:
                        create_time = order['create_time']
                        if create_time > month:
                            if 'einvoice_copy' in order['invoice']:
                                url = '{0}/cgi-bin/filetransfer?action=multimedia&ticket_id={1}&ticket={2}&id={3}&token={4}&lang=zh_CN'.format(self.url, self.gh_id, ticket, order['invoice']['einvoice_copy'], self.token)
                                resp = self.session.get(url, verify=False)
                                with open('认证发票/{0}{1}.pdf'.format(nick_name, utils.unix_to_date(create_time)), 'wb') as f:
                                    f.write(resp.content)
                                order['download'] = '已下'
                            order_content.append(order)                            
                    content = order_content
                else:
                    self.log_err('获取发票信息失败')
                    self.debug_log(data)
            else:
                self.log_err('未发现发票信息')
            self.save_data('invoice_info.json', utils.get_today(), content)                    
        except:
            self.debug_log(res.content)


# 主爬取方法
def collect(login_service, account, data_date=None, skip_exist=True):
    if not data_date:
        data_date = utils.get_yesterday()
    srv = WeChatClient(login_service)
    if not srv.enter(account['gh_id'], account['nickname'], account['pwd']):
        return
    account_type = srv.detect_account_type()
    if account_type:
        srv.log_line(account_type)
        account['type'] = account_type
    account['nickname'], account['wx_id'], account['gs'] = srv.get_nickname_when_not()
    srv.save_basic_info(account, data_date)
    srv.save_messages_detail_info(data_date)
    srv.detect_or_save('app-msg-all.json', '单篇图文详细数', data_date,
                       srv.save_messages_all_info, skip_exist)
    srv.detect_or_save('user-counter.json', '关注统计数', data_date,
                       srv.save_subscriber_info, skip_exist)
    srv.detect_or_save('user-attr.json', '用户属性统计数', data_date,
                       srv.save_subscriber_attr, skip_exist)
    srv.detect_or_save('app-msg-counter.json', '图文统计数',
                       data_date, srv.save_messages_info, skip_exist)
    srv.detect_or_save('publisher.json', '流量主统计数', data_date,
                       srv.save_publisher_info, skip_exist=False, verbose=False)
    srv.save_violation_info()
    srv.save_advertiser_info()


def test_collect(date):
    library = utils.default_library()
    account_list = library.get_account_list()
    if account_list:
        login_srv = use_default_login()
        for gh_id in account_list:
            account = library.get_account_data(gh_id)
            collect(login_srv, account, date)
    else:
        print('账号列表为空，退出')


# 爬取迁移状态线程
class GetTransferInfo(threading.Thread):
    def __init__(self, srv, data, date):
        threading.Thread.__init__(self)
        self.service = srv
        self.account = data
        self.date = date

    def run(self):
        srv = WeChatClient(self.service)
        if not srv.enter(self.account['gh_id'], self.account['nickname'], self.account['pwd']):
            return
        srv.save_basic_info(self.account, self.date)
        srv.get_transfer_info(self.date)


# 爬取流量主线程
class CollectPublisherThread(threading.Thread):
    def __init__(self, srv, data, start_date, end_date):
        threading.Thread.__init__(self)
        self.service = srv
        self.account = data
        self.start_date = start_date
        self.end_date = end_date

    def run(self):
        srv = WeChatClient(self.service)
        if not srv.enter(self.account['gh_id'], self.account['nickname'], self.account['pwd']):
            return
        account_type = srv.detect_account_type()
        if account_type:
            srv.log_line(account_type)
            self.account['type'] = account_type
        self.account['nickname'], self.account['wx_id'], self.account['gs'] = srv.get_nickname_when_not()
        srv.save_basic_info(self.account, self.end_date)
        srv.get_publisher_info(self.start_date, self.end_date)


 # 主爬取线程
class CollectThread(threading.Thread):
    def __init__(self, srv, data, date=None, skip_exist=True):
        threading.Thread.__init__(self)
        self.service = srv
        self.account = data
        self.date = date
        self.skip_exist = skip_exist

    def run(self):
        collect(self.service, self.account, self.date, self.skip_exist)


if __name__ == '__main__':
    test_collect(input('> 请输入所需动态数据日期（默认为昨天）: '))
